// OpentTxl-C Version 11 parse tree unparser
// J.R. Cordy, Jan 2023

// Copyright 2023, James R. Cordy and others

// Permission is hereby granted, free of charge, to any person obtaining a copy of this software 
// and associated documentation files (the “Software”), to deal in the Software without restriction, 
// including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, 
// and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, 
// subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all copies 
// or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 
// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE 
// AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

// The OpenTxl parse tree unparser.
// Unparses parse trees to output text using a depth-first traversal.

// Modification Log

// v11.0 Initial revision, adapted from OpenTxl 11.0

// v11.3 Corrected spacing bug for empty output tokens

// I/O, strings, memory allocation
#include "support.h"

// Global modules
#include "limits.h"
#include "options.h"
#include "tokens.h"
#include "trees.h"
#include "errors.h"
#include "shared.h"
#include "charset.h"
#include "idents.h"

// Compiler modules, for parse tree output
#include "rules.h"
#include "scan.h"

// Check interface consistency
#include "unparse.h"

// Indent limt
static int unparser_maxIndent; // min (indentLimit, options.outputLineLength div 2)

// Routines for printing parse and grammar trees
static int unparser_nparselines;                       // 0
static struct ruleLocalsT *unparser_localVarsAddress;  // 0

static void unparser_realPrintParse (const treePT parseTP, const int outstream, const int indentation);

#ifndef STANDALONE
static bool unparser_emptyTree (const treePT parseTP)
{
    return ((tree_trees[parseTP].kind == treeKind_empty) || ((((tree_trees[parseTP].kind == treeKind_repeat) 
        || (tree_trees[parseTP].kind == treeKind_list)) || (tree_trees[parseTP].kind == treeKind_choose)) 
            && (tree_trees[tree_kids[tree_trees[parseTP].kidsKP]].kind == treeKind_empty)));
}

static void unparser_printKids (const int kidCount, const kidPT kidsKP, const int outstream, const int indentation, const bool internalTree)
{
    for (int k = 0; k <= kidCount - 1; k++) {
        const treePT kidTP = tree_kids[kidsKP + k];

        // Elide formatting cues when printing trees
        if ((!unparser_emptyTree (kidTP)) || ((outstream >= tfstderr) 
                && ((tree_trees[kidTP].name == empty_T) || (options_option[verbose_p])))) {

            if (((kidCount > 1) || (indentation == 1)) && (tree_trees[kidTP].kind != treeKind_literal)) {
                fprintf (tffile (outstream), "\n");
                if (!(options_option[darren_p])) {
                    string indentblanks;
                    substring (indentblanks, BLANKS, 1, indentation);
                    fprintf (tffile (outstream), "%s", indentblanks);
                }
                unparser_nparselines += 1;
            } else if ((kidCount > 1) || (indentation == 1)) {
                fprintf (tffile (outstream), " ");
            }

            unparser_realPrintParse (kidTP, outstream, indentation);
        }
    }
}

static void unparser_printRepeatKids (const kidPT originalKidsKP, const int outstream, const int indentation, const bool internalTree)
{
    int kidsKP = originalKidsKP;
    while (true) {
        const treePT kidTP = tree_kids[kidsKP];
        const treePT moreKidsTP = tree_kids[kidsKP + 1];

        if ((!unparser_emptyTree (kidTP)) || ((outstream >= tfstderr) 
                && (((tree_trees[kidTP].name) == empty_T) || (options_option[verbose_p])))) {

            if (((tree_kidCount > 1) || (indentation == 1)) && (tree_trees[kidTP].kind != treeKind_literal)) {
                fprintf (tffile (outstream), "\n");
                if (!(options_option[darren_p])) {
                    string indentblanks;
                    substring (indentblanks, BLANKS, 1, indentation);
                    fprintf (tffile (outstream), "%s", indentblanks);
                }
                unparser_nparselines += 1;
            }
            unparser_realPrintParse (kidTP, outstream, indentation);
        }

        if (tree_trees[moreKidsTP].kind == treeKind_empty) break;

        kidsKP = tree_trees[moreKidsTP].kidsKP;
    }
}

static void unparser_underscore (const string s, string result)
{
    stringcpy (result, s);
    while (true) {
        int spindex = stringindex (result, " ");
        if (spindex == 0) break;
        lstringchar (result, spindex) = '_';
    }

    if (stringncmp (result, "opt_'", 5) == 0) {
        stringcpy (result, "opt_literal");
    } else if (stringncmp (result, "lit_'", 5) == 0) {
        stringcpy (result, "literal");
    }
}

static void unparser_printTypedTree (const treePT parseTP, const int outstream, const bool grammar_p, const bool end_p)
{
    if ((tree_trees[parseTP].kind == treeKind_empty) && (outstream < tfstderr)) {
        // Elide empty in XML output
        return;
    }

    tokenT treeT = tree_trees[parseTP].rawname;

    if (tree_trees[parseTP].kind == treeKind_ruleCall) {
        treeT = rule_rules[treeT].name;
    }

    longstring *treeName = ident_idents[treeT];

    if (tree_trees[parseTP].kind == treeKind_literal) {
        if (end_p) {
            return;
        }
    } else {
        if (!end_p) {
            fprintf (tffile (outstream), "<");
        } else {
            if ((tree_trees[parseTP].kind == treeKind_empty) 
                    || ((tree_trees[parseTP].kind >= treeKind_firstTime) && (tree_trees[parseTP].kind != treeKind_ruleCall))) {
                return;
            }
            fprintf (tffile (outstream), "</");
        }
    }

    switch (tree_trees[parseTP].kind) {
        case treeKind_order: case treeKind_choose: case treeKind_repeat: case treeKind_list: case treeKind_leftchoose: 
        case treeKind_generaterepeat: case treeKind_generatelist: case treeKind_lookahead:
            {
                string exttype, usexttype;
                externalType ((*treeName), exttype);
                unparser_underscore (exttype, usexttype);
                fprintf (tffile (outstream), "%s", usexttype);
            }
            break;

        case treeKind_empty:
            {
                fprintf (tffile (outstream), "%s", (*treeName));
            }
            break;

        case treeKind_firstTime:
            {
                fprintf (tffile (outstream), "varbind");

                if (!end_p) {
                    fprintf (tffile (outstream), " name=\"%s\"", (*treeName));
                    const struct ruleLocalsT *localVars = unparser_localVarsAddress;
                    string exttype, usexttype;
                    externalType (*ident_idents[rule_ruleLocals[localVars->localsBase + tree_trees[parseTP].count].typename_], exttype);
                    unparser_underscore (exttype, usexttype);
                    fprintf (tffile (outstream), " type=\"%s\"", usexttype);
                }
            }
            break;

        case treeKind_subsequentUse:
        case treeKind_expression:
        case treeKind_lastExpression:
            {
                fprintf (tffile (outstream), "varref");

                if (!end_p) {
                    if ((tree_trees[parseTP].kind == treeKind_lastExpression) && (options_option[verbose_p])) {
                        fprintf (tffile (outstream), " last=\"true\"");
                    }
                    fprintf (tffile (outstream), " name=\"%s\"", (*treeName));
                    const struct ruleLocalsT *localVars = unparser_localVarsAddress;
                    string exttype, usexttype;
                    externalType (*ident_idents[rule_ruleLocals[localVars->localsBase + tree_trees[parseTP].count].typename_], exttype);
                    unparser_underscore (exttype, usexttype);
                    fprintf (tffile (outstream), " type=\"%s\"", usexttype);
                }
            }
            break;

        case treeKind_ruleCall:
            {
                fprintf (tffile (outstream), "rulecall");
                if (!end_p) {
                    fprintf (tffile (outstream), " name=\"%s\"", (*treeName));
                }
            }
            break;

        case treeKind_literal:
            {
                charset_putXmlCode (outstream, (*treeName));
            }
            break;

        default :
            {
                if ((tree_trees[parseTP].kind > treeKind_literal) && (tree_trees[parseTP].kind <= lastUserTokenKind)) {
                    fprintf (tffile (outstream), "%s", *ident_idents[kindType[tree_trees[parseTP].kind]]);
                    if ((!grammar_p) && (!end_p)) {
                        fprintf (tffile (outstream), ">");
                        charset_putXmlCode (outstream, (*treeName));
                    }
                } else {
                    fprintf (tffile (outstream), "ILLEGAL");
                    if (!end_p) {
                        fprintf (tffile (outstream), " kind=\"%d\">%s", tree_trees[parseTP].kind, (*treeName));
                    }
                }
            }
            break;
    }

    if (tree_trees[parseTP].kind != treeKind_literal) {
        if (!end_p) {
            if ((tree_trees[parseTP].kind == treeKind_empty) || ((tree_trees[parseTP].kind >= treeKind_firstTime) && (tree_trees[parseTP].kind != treeKind_ruleCall))) {
                fprintf (tffile (outstream), "/");
            }
            if (((tree_trees[parseTP].kind <= treeKind_literal) || (tree_trees[parseTP].kind > lastUserTokenKind)) && (!grammar_p)) {
                fprintf (tffile (outstream), ">");
            }
        } else {
            fprintf (tffile (outstream), ">");
        }
    }
}
#endif

static void unparser_realPrintParse (const treePT parseTP, const int outstream, const int indentation)
{
#ifndef STANDALONE
    if (parseTP == nilTree) {
        return;
    }

    tokenT treeT = tree_trees[parseTP].rawname;

    if (tree_trees[parseTP].kind == treeKind_ruleCall) {
        treeT = rule_rules[treeT].name;
    }

    const string *treeName = (string *) ident_idents[treeT];

    const bool internalTree = (tree_trees[parseTP].kind < firstLiteralKind) 
        && (((stringchar (*treeName, 1) == '_') && (stringchar (*treeName, 2) == '_')) || (tree_trees[parseTP].kind == treeKind_lookahead));

    int indent = 0;
    int nKids = 0;

    if (indentation == 0) {
        unparser_nparselines = 0;
    }

    if (!internalTree) {
        if (indentation < unparser_maxIndent) {
            indent = 1;
        }
        unparser_printTypedTree (parseTP, outstream, false, false);
    }

    const int oldparselines = unparser_nparselines;

    switch (tree_trees[parseTP].kind) {
        case treeKind_repeat: case treeKind_list:
            {
                if (unparser_localVarsAddress != 0) {
                    unparser_printKids (2, tree_trees[parseTP].kidsKP, outstream, indentation + indent, internalTree);
                } else {
                    unparser_printRepeatKids (tree_trees[parseTP].kidsKP, outstream, indentation + indent, internalTree);
                }
            }
            break;

        case treeKind_order: case treeKind_generatelist: case treeKind_lookahead:
            {
                nKids = tree_trees[parseTP].count;
                unparser_printKids (nKids, tree_trees[parseTP].kidsKP, outstream, indentation + indent, internalTree);
            }
            break;

        case treeKind_choose: case treeKind_leftchoose: case treeKind_generaterepeat:
            {
                nKids = 1;
                unparser_printKids (1, tree_trees[parseTP].kidsKP, outstream, indentation + indent, internalTree);
            }
            break;

        case treeKind_expression: case treeKind_lastExpression: case treeKind_ruleCall:
            {
                if (tree_trees[parseTP].kidsKP != nilKid) {
                    nKids = 0;
                    for (int c = tree_trees[parseTP].kidsKP; c <= maxKids; c++) {
                        if ((tree_kids[c]) == 0) break;
                        nKids += 1;
                    }
                    unparser_printKids (nKids, tree_trees[parseTP].kidsKP, outstream, indentation + indent, internalTree);
                }
            }
            break;

        default :
            break;
    }

    if (!internalTree) {
        if ((nKids > 1) || (unparser_nparselines > oldparselines)) {
            fprintf (tffile (outstream), "\n");
            unparser_nparselines += 1;
            if ((indentation > 0) && (!(options_option[darren_p]))) {
                string indentblanks;
                substring (indentblanks, BLANKS, 1, indentation);
                fprintf (tffile (outstream), "%s", indentblanks);
            }
        }
        if (indentation > 0) {
            unparser_printTypedTree (parseTP, outstream, false, true);
        } else {
            unparser_printTypedTree (parseTP, outstream, false, true);
            fprintf (tffile (outstream), "\n");
        }
    }
#else
    error ("", "XML output disabled in standalone applications", FATAL, 951);
#endif
}

void unparser_printParse (const treePT parseTP, const int outstream, const int indentation)
{
    unparser_localVarsAddress = (struct ruleLocalsT *) 0;
    unparser_realPrintParse (parseTP, outstream, indentation);
}

#ifndef STANDALONE
void unparser_printPatternParse (const treePT parseTP, const struct ruleLocalsT *localVars, const int indentation)
{
    unparser_localVarsAddress = (struct ruleLocalsT *) (&(*localVars));
    unparser_realPrintParse (parseTP, 0, indentation);
}
#endif

#ifndef NOCOMPILE
// 1-indexed [1 .. maxSymbols]
static array (treePT, unparser_grammarList);
static int unparser_grammarLength;     // 0
#endif

static void unparser_real_printGrammar (const treePT grammarTP, const int indentation);

#ifndef NOCOMPILE
static void unparser_printGrammarKids (const int kidCount, const kidPT kidsKP, const int indentation, const enum treeKindT kind, const bool internalTree)
{
    for (int k = 0; k <= kidCount - 1; k++) {
        if (kidCount > 1) {
            fprintf (stderr, "\n");
            string indentblanks;
            substring (indentblanks, BLANKS, 1, indentation);
            fprintf (stderr, "%s", indentblanks);
        } else {
            const enum treeKindT kidKind = tree_trees[tree_kids[kidsKP + k]].kind;
            if (kidKind < treeKind_literal) {
                fprintf (stderr, "\n");
                string indentblanks;
                substring (indentblanks, BLANKS, 1, indentation);
                fprintf (stderr, "%s", indentblanks);
            }
        }
        unparser_real_printGrammar (tree_kids[kidsKP + k], indentation);
    }
}
#endif

static void unparser_real_printGrammar (const treePT grammarTP, const int indentation)
{
#ifndef NOCOMPILE
    if (grammarTP == nilTree) {
        return;
    }

    tokenT treeT = tree_trees[grammarTP].rawname;

    if (tree_trees[grammarTP].kind == treeKind_ruleCall) {
        // name field actually encodes rule index
        treeT = rule_rules[treeT].name;
    }

    const string *treeName = (string *) ident_idents[treeT];

    const bool internalTree = (tree_trees[grammarTP].kind < firstLiteralKind) 
        && (((stringchar (*treeName, 1) == '_') && (stringchar (*treeName, 2) == '_')) ||
            ((stringchar (*treeName, 4) == '_') && (stringchar (*treeName, 5) == '_')));

    int indent = 0;
    int nKids = 0;

    if (indentation < unparser_maxIndent) {
        indent = 2;
    }

    unparser_printTypedTree (grammarTP, tfstderr, true, false);

    if ((tree_trees[grammarTP].kind == treeKind_choose) 
            || (tree_trees[grammarTP].kind == treeKind_order) 
            || (tree_trees[grammarTP].kind == treeKind_repeat) 
            || (tree_trees[grammarTP].kind == treeKind_list) 
            || (tree_trees[grammarTP].kind == treeKind_leftchoose) 
            || (tree_trees[grammarTP].kind == treeKind_generaterepeat) 
            || (tree_trees[grammarTP].kind == treeKind_generatelist) 
            || (tree_trees[grammarTP].kind == treeKind_lookahead)) { 

        // if we've already printed it once, then reference the original nonterminal label
        for (int g = unparser_grammarLength; g >= 1; g--) {
            if (grammarTP == unparser_grammarList[g]) {
                fprintf (stderr, " ref=\"%d\"/>", g);
                return;
            }
        }

        unparser_grammarLength += 1;
        unparser_grammarList[unparser_grammarLength] = grammarTP;

        // make the kind of the new nonterminal clear
        enum treeKindT kind = tree_trees[grammarTP].kind;
        if (!(options_option[verbose_p])) {
            if (kind == treeKind_leftchoose) {
                kind = treeKind_choose;
            } else if (kind == treeKind_generaterepeat) {
                kind = treeKind_repeat;
            } else if (kind == treeKind_generatelist) {
                kind = treeKind_list;
            }
        }
        assert (lstringchar (*ident_idents[kindType[kind]], 1) == '*');

        fprintf (stderr, " kind=\"%s\"", * (string *) &(lstringchar (*ident_idents[kindType[kind]], 2)));

        // label the new nonterminal for future reference
        if (!internalTree) {
            fprintf (stderr, " label=\"%d\"", unparser_grammarLength);
        }
    }

    if ((tree_trees[grammarTP].kind) != treeKind_literal) {
        fprintf (stderr, ">");
    }

    switch (tree_trees[grammarTP].kind) {
        case treeKind_order: case treeKind_repeat: case treeKind_list:
            {
                nKids = tree_trees[grammarTP].count;
                unparser_printGrammarKids (nKids, tree_trees[grammarTP].kidsKP, indentation + indent, treeKind_order, internalTree);
            }
            break;

        case treeKind_choose: case treeKind_leftchoose: case treeKind_generaterepeat: case treeKind_generatelist: case treeKind_lookahead:
            {
                nKids = tree_trees[grammarTP].count;
                unparser_printGrammarKids (nKids, tree_trees[grammarTP].kidsKP, indentation + indent, treeKind_choose, internalTree);
            }
            break;

        case treeKind_expression: case treeKind_lastExpression: case treeKind_ruleCall:
            {
                if (tree_trees[grammarTP].kidsKP != nilKid) {
                    nKids = 0;
                    for (int c = tree_trees[grammarTP].kidsKP; c <= maxKids; c++) {
                        if ((tree_kids[c]) == 0) break;
                        nKids += 1;
                    }
                    unparser_printGrammarKids (nKids, tree_trees[grammarTP].kidsKP, indentation + indent, treeKind_expression, internalTree);
                }
            }
            break;

        default :
            break;
    }

    if (nKids > 1) {
        fprintf (stderr, "\n");
        string indentblanks;
        substring (indentblanks, BLANKS, 1, indentation);
        fprintf (stderr, "%s", indentblanks);
        unparser_printTypedTree (grammarTP, 0, true, true);

    } else if (indentation > 0) {
        unparser_printTypedTree (grammarTP, 0, true, true);

    } else {
        fprintf (stderr, "\n");
        unparser_printTypedTree (grammarTP, 0, true, true);
        fprintf (stderr, "\n");
    }
#endif
}

void unparser_printGrammar (const treePT grammarTP, const int indentation)
{
#ifndef NOCOMPILE
    unparser_grammarLength = 0;
    unparser_real_printGrammar (grammarTP, indentation);
#endif
}


// Routines for deparsing output

// Output paragraphing parameters
static int unparser_tempIndent;        // max (1, options_indentIncrement / 2)
static int unparser_indent;            // 0

// Output line buffer
// the + maxStringLength + 1 is necessary for type cheats
// use variable buffer size to force dynamic allocation for efficiency
static int unparser_outputLineBufferSize;  // maxLineLength + maxStringLength + 1
static array (char, unparser_outputline);
static array (char, unparser_savedoutputline);
static int unparser_lineLength;  // 0

// Output state
static bool unparser_emptyLine;  // true
static bool unparser_blankLine;  // true
static bool unparser_spacing;    // true
static char unparser_lastLeafNameEnd;  // ' '

static void unparser_outputindent (const int n)
{
    // N.B. Cannot use Turing substrings here since maxIndent can be > 4095
    if (n <= unparser_maxIndent) {
        for (int i = 1; i <= n; i++) {
            lstringchar (unparser_outputline, i) = ' ';
        }
        lstringchar (unparser_outputline, n + 1) = EOS;
        unparser_lineLength = n;
    } else {
        for (int i = 1; i <= unparser_maxIndent; i++) {
            lstringchar (unparser_outputline, i) = ' ';
        }
        lstringchar (unparser_outputline, unparser_maxIndent + 1) = EOS;
        unparser_lineLength = unparser_maxIndent;
    }
}

static void unparser_printLeavesTraversal (const treePT subtreeTP, const int outstream)
{
    // Stack use limitation - to avoid crashes
    checkstack ();

    switch (tree_trees[subtreeTP].kind) {

        case treeKind_choose:
            // Optimize by skipping choose chains
            {
                treePT chainTP = tree_kids[tree_trees[subtreeTP].kidsKP];
                while (true) {
                    if ((tree_trees[chainTP].kind) != treeKind_choose) break;
                    chainTP = tree_kids[tree_trees[chainTP].kidsKP];
                }
                unparser_printLeavesTraversal (chainTP, outstream);
            }
            break;

        case treeKind_order:
            // Print out the kids
            {
                kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                for (int i = 1; i <= tree_trees[subtreeTP].count; i++) {
                    // We don't print attributes unless asked
                    if ((tree_trees[tree_kids[subtreeKidsKP]].name == ATTR_T) && (!(options_option[attr_p]))) break;
                    unparser_printLeavesTraversal (tree_kids[subtreeKidsKP], outstream);
                    subtreeKidsKP += 1;
                }
            }
            break;

        case treeKind_repeat:
            {
                kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                assert (subtreeKidsKP != nilKid);
                while (true) {
                    unparser_printLeavesTraversal (tree_kids[subtreeKidsKP], outstream);
                    if (tree_trees[tree_kids[subtreeKidsKP + 1]].kind == treeKind_empty) break;
                    subtreeKidsKP = tree_trees[tree_kids[subtreeKidsKP + 1]].kidsKP;
                }
            }
            break;

        case treeKind_list:
            {
                kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                assert (subtreeKidsKP != nilKid);
                if ((tree_trees[tree_kids[subtreeKidsKP + 1]].kind) != treeKind_empty) {
                    while (true) {
                        unparser_printLeavesTraversal (tree_kids[subtreeKidsKP], outstream);
                        subtreeKidsKP = tree_trees[tree_kids[subtreeKidsKP + 1]].kidsKP;
                        if (tree_trees[tree_kids[subtreeKidsKP + 1]].kind == treeKind_empty) break;
                        unparser_printLeavesTraversal (commaTP, outstream);
                    }
                }
            }
            break;

        case treeKind_empty:
            {
                const tokenT name = tree_trees[subtreeTP].name;

                if ((name != empty_T) && (name != ATTR_T)) {
                    if (name == NL_T) {
                        if ((!unparser_emptyLine) || (!unparser_blankLine)) {
                            fprintf (tffile (outstream), "%s\n", unparser_outputline);
                            lstringcpy (unparser_outputline, "");
                            unparser_blankLine = unparser_emptyLine;
                            unparser_emptyLine = true;
                            unparser_lineLength = 0;
                        }
                    } else if (name == FL_T) {
                        if (!unparser_emptyLine) {
                            fprintf (tffile (outstream), "%s\n", unparser_outputline);
                            lstringcpy (unparser_outputline, "");
                            unparser_blankLine = unparser_emptyLine;
                            unparser_emptyLine = true;
                            unparser_lineLength = 0;
                        }
                    } else if (name == IN_T) {
                        unparser_indent += options_indentIncrement;
                    } else if (name == EX_T) {
                        if (unparser_indent >= options_indentIncrement) {
                            unparser_indent -= options_indentIncrement;
                        } else {
                            unparser_indent = 0;
                        }
                    } else if (name == SP_T) {
                        if (unparser_lineLength < options_outputLineLength) {
                            lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                            unparser_lineLength += 1;
                        } else {
                            // this line is full, and will force a new line at the next token anyway
                        }
                    } else if (name == TAB_T) {
                        const int newLineLength = unparser_lineLength + (8 - (unparser_lineLength % 8));
                        if (newLineLength > options_outputLineLength) {
                            lsubstring (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), 
                                BLANKS, 1, options_outputLineLength - unparser_lineLength);
                            unparser_lineLength = options_outputLineLength;
                        } else {
                        lsubstring (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), 
                            BLANKS, 1, 8 - (unparser_lineLength % 8));
                        unparser_lineLength = newLineLength;
                    }
                    } else if (name == SPOFF_T) {
                        unparser_spacing = false;
                    } else if (name == SPON_T) {
                        unparser_spacing = true;
                    } else if ((name == KEEP_T) || (name == FENCE_T) || (name == SEE_T) || (name == NOT_T)) {
                        // Nothing to do
                    } else {
                        // Custom TAB_, IN_ or EX_ formatting symbol
                        longstring *subtreeName = (longstring *)ident_idents[name];
                        assert ((stringlen (*subtreeName) >= 4) && ((stringncmp (subtreeName, "TAB_", 4) == 0) 
                            || (stringncmp (subtreeName, "IN_", 3) == 0) || (stringncmp (subtreeName, "EX_", 3) == 0)));
                        // Get custom value
                        const char subtreeName1 = lstringchar (*subtreeName, 1);
                        int numindex = 4;
                        if (subtreeName1 == 'T') {
                            numindex += 1;
                        }
                        int value = 0;
                        for (int i = numindex; i <= stringlen (*subtreeName); i++) {
                            char d = lstringchar (*subtreeName, i);
                            if (!(charset_digitP[(unsigned char) d])) break;
                            value = (value * 10) + (d - '0');
                        }
                        if (value > unparser_maxIndent) {
                            value = unparser_maxIndent;
                        }
                        // implement action
                        if (value != 0) {
                            if (subtreeName1 == 'T') {
                                // Custom tabstop
                                if (unparser_lineLength < value - 1) {
                                    lsubstring (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), 
                                        BLANKS, 1, (value - unparser_lineLength) - 1);
                                    unparser_lineLength = value - 1;
                                } else if (options_option[notabnl_p]) {
                                    if (unparser_lineLength == value - 1) {
                                        // Already exactly where we should be
                                    } else if (unparser_lineLength < options_outputLineLength) {
                                        lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                                        unparser_lineLength += 1;
                                    } else {
                                        // this line is full, and will force a new line at the next token anyway
                                    }
                                } else {
                                    if (!unparser_emptyLine) {
                                        fprintf (tffile (outstream), "%s\n", unparser_outputline);
                                        unparser_blankLine = false;
                                    }
                                    lsubstring (unparser_outputline, BLANKS, 1, (value - 1));
                                    unparser_lineLength = value - 1;
                                }
                                unparser_emptyLine = false;
                                unparser_lastLeafNameEnd = ' ';

                            } else if (subtreeName1 == 'I') {
                                // Custom indent
                                unparser_indent += value;

                            } else if (subtreeName1 == 'E') {
                                // Custom exdent
                                if (unparser_indent > value) {
                                    unparser_indent -= value;
                                } else {
                                    unparser_indent = 0;
                                }
                            }
                        }
                    }
                }
            }
            break;

        case treeKind_newline:
            {
                fprintf (tffile (outstream), "%s\n", unparser_outputline);
                lstringcpy (unparser_outputline, "");
                unparser_blankLine = unparser_emptyLine;
                unparser_emptyLine = true;
                unparser_lineLength = 0;
            }
            break;

        default :
            {
                // it's a leaf - print it out
                longstring *leafName = (longstring *) (ident_idents[tree_trees[subtreeTP].rawname]);
                const int lengthLeaf = lstringlen ((*leafName));

                // Ignore empty leaves completely
                if (lengthLeaf > 0) {
                    const char leafName1 = lstringchar (*leafName, 1);

                    if (unparser_emptyLine) {
                        unparser_outputindent (unparser_indent);
                    } else {
                        if (((unparser_lineLength + 1) + lengthLeaf) > options_outputLineLength) {
                            if ((options_option[raw_p]) || (options_option[newline_p])) {
                                fprintf (tffile (outstream), "%s", unparser_outputline);
                                unparser_outputindent (0);
                                unparser_blankLine = false;
                                if (unparser_spacing) {
                                    if (!(options_option[raw_p])) {
                                        if ((charset_spaceAfterP[(unsigned char) unparser_lastLeafNameEnd]) && ((charset_spaceBeforeP[(unsigned char) leafName1]) || ((tree_trees[subtreeTP].kind) == treeKind_number))) {
                                            lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                                            unparser_lineLength += 1;
                                        }
                                    } else {
                                        if (((charset_idP[(unsigned char) unparser_lastLeafNameEnd]) && (charset_idP[(unsigned char) leafName1])) && (!(options_option[charinput_p]))) {
                                            lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                                            unparser_lineLength += 1;
                                        }
                                    }
                                }
                            } else {
                                if (!unparser_spacing) {
                                    int i = unparser_lineLength;
                                    for (;;) {
                                        if ((i == 0) || (lstringchar (unparser_outputline, i) == ' ')) break;
                                        i -= 1;
                                    }
                                    if (i < (options_outputLineLength / 2)) {
                                        error ("", "Forced to split [SPOFF] output at line boundary", WARNING, 952);
                                        i = unparser_lineLength;
                                    }
                                    const char savedoutputlineip1 = lstringchar (unparser_outputline, i + 1);
                                    lstringchar (unparser_outputline, i + 1) = EOS;
                                    fprintf (tffile (outstream), "%s\n", unparser_outputline);
                                    lstringchar (unparser_outputline, i + 1) = savedoutputlineip1;
                                    lsubstring (unparser_savedoutputline, unparser_outputline, i + 1, unparser_lineLength);
                                    unparser_outputindent ((unparser_indent + unparser_tempIndent));
                                    unparser_blankLine = false;
                                    lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), unparser_savedoutputline);
                                    unparser_lineLength = lstringlen (unparser_outputline);
                                } else {
                                    fprintf (tffile (outstream), "%s\n", unparser_outputline);
                                    unparser_outputindent ((unparser_indent + unparser_tempIndent));
                                    unparser_blankLine = false;
                                }
                            }
                        } else {
                            if (unparser_spacing) {
                                if (!(options_option[raw_p])) {
                                    if ((charset_spaceAfterP[(unsigned char) unparser_lastLeafNameEnd]) && ((charset_spaceBeforeP[(unsigned char) leafName1]) || ((tree_trees[subtreeTP].kind) == treeKind_number))) {
                                        lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                                        unparser_lineLength += 1;
                                    }
                                } else {
                                    if (((charset_idP[(unsigned char) unparser_lastLeafNameEnd]) && (charset_idP[(unsigned char) leafName1])) && (!(options_option[charinput_p]))) {
                                        lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                                        unparser_lineLength += 1;
                                    }
                                }
                            }
                        }
                    }

                    if (lengthLeaf > options_outputLineLength) {
                        if ((!(options_option[raw_p])) && (!(options_option[quiet_p]))) {
                            error ("", "Output token too long for output width", WARNING, 953);
                        }
                        fprintf (tffile (outstream), "%s\n", (*leafName));
                        unparser_blankLine = false;
                        unparser_outputindent (unparser_indent);
                    } else {
                        if ((unparser_lineLength + lengthLeaf) > options_outputLineLength) {
                            unparser_outputindent ((options_outputLineLength - lengthLeaf));
                        }
                        lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), (*leafName));
                        unparser_lineLength += lengthLeaf;
                        unparser_emptyLine = false;
                        if (unparser_lineLength > 0) {
                            unparser_lastLeafNameEnd = lstringchar (unparser_outputline, unparser_lineLength);
                        } else {
                            unparser_lastLeafNameEnd = EOS;
                        }
                        if ((unparser_lastLeafNameEnd == '\r') || (unparser_lastLeafNameEnd == '\n')) {
                            fprintf (tffile (outstream), "%s", unparser_outputline);
                            unparser_blankLine = false;
                            unparser_outputindent (unparser_indent);
                        }
                    }
                }
                break;
            }
    }

    assert ((unparser_lineLength == lstringlen (unparser_outputline)) && (unparser_lineLength <= options_outputLineLength));
}

static void unparser_printMatchTraversal (const treePT subtreeTP, const treePT matchtreeTP, const int outstream)
{
    if (subtreeTP == matchtreeTP) {

        if ((unparser_lineLength + 6) > options_outputLineLength) {
            fprintf (tffile (outstream), "%s\n", unparser_outputline);
            unparser_blankLine = false;
            unparser_outputindent (unparser_indent + unparser_tempIndent);
        }

        if (charset_spaceAfterP[(unsigned char) unparser_lastLeafNameEnd] && (!unparser_blankLine)) {
            lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
            unparser_lineLength += 1;
            unparser_lastLeafNameEnd = ' ';
        }

        lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), "|>>>|");
        unparser_emptyLine = false;
        unparser_lineLength += 5;

        unparser_printLeavesTraversal (subtreeTP, outstream);

        if ((unparser_lineLength + 5) > options_outputLineLength) {
            fprintf (tffile (outstream), "%s\n", unparser_outputline);
            unparser_blankLine = false;
            unparser_outputindent (unparser_indent + unparser_tempIndent);
        }

        lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), "|<<<|");
        unparser_lineLength += 5;
        unparser_emptyLine = false;

    } else {

        switch (tree_trees[subtreeTP].kind) {

            case treeKind_choose:
                {
                    unparser_printMatchTraversal (tree_kids[tree_trees[subtreeTP].kidsKP], matchtreeTP, outstream);
                }
                break;

            case treeKind_order:
                {
                    kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                    for (int i = 1; i <= tree_trees[subtreeTP].count; i++) {
                        if (((tree_trees[tree_kids[subtreeKidsKP]].name) == ATTR_T) && (!(options_option[attr_p]))) break;
                        unparser_printMatchTraversal (tree_kids[subtreeKidsKP], matchtreeTP, outstream);
                        subtreeKidsKP += 1;
                    }
                }
                break;

            case treeKind_repeat:
                {
                    kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                    assert (subtreeKidsKP != nilKid);
                    while (true) {
                        unparser_printMatchTraversal (tree_kids[subtreeKidsKP], matchtreeTP, outstream);
                        if (tree_trees[tree_kids[subtreeKidsKP + 1]].kind == treeKind_empty) break;
                        subtreeKidsKP = tree_trees[tree_kids[subtreeKidsKP + 1]].kidsKP;
                    }
                }
                break;

            case treeKind_list:
                {
                    kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                    assert (subtreeKidsKP != nilKid);
                    if (tree_trees[tree_kids[subtreeKidsKP + 1]].kind != treeKind_empty) {
                        while (true) {
                            unparser_printMatchTraversal (tree_kids[subtreeKidsKP], matchtreeTP, outstream);
                            subtreeKidsKP = tree_trees[tree_kids[subtreeKidsKP + 1]].kidsKP;
                            if (tree_trees[tree_kids[subtreeKidsKP + 1]].kind == treeKind_empty) break;
                            unparser_printMatchTraversal (commaTP, matchtreeTP, outstream);
                        }
                    }
                }
                break;

            case treeKind_empty:
                {
                    const tokenT name = tree_trees[subtreeTP].name;

                    if ((name != empty_T) && (name != ATTR_T)) {
                        // Might be a paragraphing symbol
                        if (name == NL_T) {
                            if ((!unparser_emptyLine) || (!unparser_blankLine)) {
                                fprintf (tffile (outstream), "%s\n", unparser_outputline);
                                lstringcpy (unparser_outputline, "");
                                unparser_blankLine = unparser_emptyLine;
                                unparser_emptyLine = true;
                                unparser_lineLength = 0;
                            }
                        } else if (name == FL_T) {
                            if (!unparser_emptyLine) {
                                fprintf (tffile (outstream), "%s\n", unparser_outputline);
                                lstringcpy (unparser_outputline, "");
                                unparser_blankLine = unparser_emptyLine;
                                unparser_emptyLine = true;
                                unparser_lineLength = 0;
                            }
                        } else if (name == IN_T) {
                            unparser_indent += options_indentIncrement;
                        } else if (name == EX_T) {
                            if (unparser_indent >= options_indentIncrement) {
                                unparser_indent -= options_indentIncrement;
                            } else {
                                unparser_indent = 0;
                            }
                        } else if (name == SP_T) {
                            if (unparser_lineLength < options_outputLineLength) {
                                lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                                unparser_lineLength += 1;
                            } else {
                                // this line is full, and will force a new line at the next token anyway
                            }
                        } else if (name == TAB_T) {
                            unparser_lineLength += 8 - (unparser_lineLength % 8);
                            if (unparser_lineLength > options_outputLineLength) {
                                // just make it a full line, to force a new line at the next token
                                lstringcpy (&(lstringchar (unparser_outputline, options_outputLineLength + 1)), "");
                                unparser_lineLength = options_outputLineLength;
                            } else {
                                lsubstring (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), 
                                    BLANKS, 1, (8 - (unparser_lineLength % 8)));
                            }
                        } else if (name == SPOFF_T) {
                            unparser_spacing = false;
                        } else if (name == SPON_T) {
                            unparser_spacing = true;
                        } else if ((((name == KEEP_T) || (name == FENCE_T)) || (name == SEE_T)) || (name == NOT_T)) {
                            // Nothing to do
                        } else {
                            // Custom TAB_, IN_ or EX_ formatting symbol
                            longstring *subtreeName = (longstring *) ident_idents[name];
                            assert ((stringlen (*subtreeName) >= 4) && ((stringncmp (subtreeName, "TAB_", 4) == 0) 
                                || (stringncmp (subtreeName, "IN_", 3) == 0) || (stringncmp (subtreeName, "EX_", 3) == 0)));
                            // Get custom value
                            const char subtreeName1 = lstringchar (*subtreeName, 1);
                            int numindex = 4;
                            if (subtreeName1 == 'T') {
                                numindex += 1;
                            }
                            int value = 0;
                            for (int i = numindex; i <= lstringlen (*subtreeName); i++) {
                                const char d = lstringchar (*subtreeName, i);
                                if (!(charset_digitP[(unsigned char) d])) break;
                                value = (value * 10) + (d - '0');
                            }
                            if (value > unparser_maxIndent) {
                                value = unparser_maxIndent;
                            }
                            // implement action
                            if (value != 0) {
                                if (subtreeName1 == 'T') {
                                    // Custom tabstop
                                    if (unparser_lineLength < value - 1) {
                                        lsubstring (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), 
                                            BLANKS, 1, (value - unparser_lineLength) - 1);
                                        unparser_lineLength = value - 1;
                                    } else if (options_option[notabnl_p]) {
                                        if (unparser_lineLength == value - 1) {
                                            // Already exactly where we should be
                                        } else if (unparser_lineLength < options_outputLineLength) {
                                            lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                                            unparser_lineLength += 1;
                                        } else {
                                            // this line is full, and will force a new line at the next token anyway
                                        }
                                    } else {
                                        if (!unparser_emptyLine) {
                                            fprintf (tffile (outstream), "%s\n", unparser_outputline);
                                            unparser_blankLine = false;
                                        }
                                        lsubstring (unparser_outputline, BLANKS, 1, (value - 1));
                                        unparser_lineLength = value - 1;
                                    }
                                    unparser_emptyLine = false;
                                    unparser_lastLeafNameEnd = ' ';

                                } else if (subtreeName1 == 'I') {
                                    // Custom indent
                                    unparser_indent += value;

                                } else if (subtreeName1 == 'E') {
                                    // Custom exdent
                                    if (unparser_indent > value) {
                                        unparser_indent -= value;
                                    } else {
                                        unparser_indent = 0;
                                    }
                                }
                            }
                        }
                    }
                }
                break;

            case treeKind_newline:
                {
                    // New line, when in -char mode
                    fprintf (tffile (outstream), "%s\n", unparser_outputline);
                    lstringcpy (unparser_outputline, "");
                    unparser_blankLine = unparser_emptyLine;
                    unparser_emptyLine = true;
                    unparser_lineLength = 0;
                }
                break;

            default :
                {
                    // it's a leaf - print it out
                    longstring *leafName = (longstring *) (ident_idents[tree_trees[subtreeTP].rawname]);
                    const int lengthLeaf = lstringlen ((*leafName));
                    const char leafName1 = lstringchar (*leafName, 1);

                    if (unparser_emptyLine) {
                        unparser_outputindent (unparser_indent);

                    } else {
                        if ((unparser_lineLength + 1) + lengthLeaf > options_outputLineLength) {
                            // must split the output line here
                            if ((options_option[raw_p]) || (options_option[newline_p])) {
                                // raw output must have no new lines unless explicitly asked for!
                                fprintf (tffile (outstream), "%s", unparser_outputline);

                                unparser_outputindent (0);
                                unparser_blankLine = false;

                                // since this line is continuing, need usual intra-line spacing
                                if (unparser_spacing) {
                                    if (!(options_option[raw_p])) {
                                        if (charset_spaceAfterP[(unsigned char) unparser_lastLeafNameEnd] 
                                                && (charset_spaceBeforeP[(unsigned char) leafName1] || (tree_trees[subtreeTP].kind) == treeKind_number)) {
                                            lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                                            unparser_lineLength += 1;
                                        }
                                    } else if (charset_idP[(unsigned char) unparser_lastLeafNameEnd] && charset_idP[(unsigned char) leafName1] 
                                            && (!(options_option[charinput_p]))) {
                                        lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                                        unparser_lineLength += 1;
                                    }
                                }

                            } else if (!unparser_spacing) {
                                // not supposed to break at the moment - must choose an appropriate previous place to break
                                int i = unparser_lineLength;
                                while (true) {
                                    if ((i == 0) || (lstringchar (unparser_outputline, i) == ' ')) break;
                                    i -= 1;
                                }
                                if (i < (options_outputLineLength / 2)) {
                                    error ("", "Forced to split [SPOFF] output at line boundary", WARNING, 952);
                                    i = unparser_lineLength;
                                }

                                // substring in place
                                const char savedoutputlineip1 = lstringchar (unparser_outputline, i + 1);
                                lstringchar (unparser_outputline, i + 1) = EOS;
                                fprintf (tffile (outstream), "%s\n", unparser_outputline);
                                lstringchar (unparser_outputline, i + 1) = savedoutputlineip1;

                                lsubstring (unparser_savedoutputline, unparser_outputline, i + 1, unparser_lineLength);

                                unparser_outputindent ((unparser_indent + unparser_tempIndent));
                                unparser_blankLine = false;

                                lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), unparser_savedoutputline);
                                unparser_lineLength = lstringlen (unparser_outputline);

                            } else {
                                fprintf (tffile (outstream), "%s\n", unparser_outputline);
                                unparser_outputindent ((unparser_indent + unparser_tempIndent));
                                unparser_blankLine = false;
                            }

                        } else {
                            if (unparser_spacing) {
                                if (!(options_option[raw_p])) {
                                    if (charset_spaceAfterP[(unsigned char) unparser_lastLeafNameEnd] 
                                            && (charset_spaceBeforeP[(unsigned char) leafName1] || (tree_trees[subtreeTP].kind == treeKind_number))) {
                                        lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                                        unparser_lineLength += 1;
                                    }
                                } else if (charset_idP[(unsigned char) unparser_lastLeafNameEnd] && charset_idP[(unsigned char) leafName1] && (!(options_option[charinput_p]))) {
                                    lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), " ");
                                    unparser_lineLength += 1;
                                }
                            }
                        }
                    }

                    if (lengthLeaf > options_outputLineLength) {
                        if ((!(options_option[raw_p])) && (!(options_option[quiet_p]))) {
                            error ("", "Output token too long for output width", WARNING, 953);
                        }
                        fprintf (tffile (outstream), "%s\n", (*leafName));
                        unparser_blankLine = false;
                        unparser_outputindent (unparser_indent);
                    } else {
                        if ((unparser_lineLength + lengthLeaf) > options_outputLineLength) {
                            // Since we already handled line overflow above, this can only mean
                            // that the indent plus the leaf is too long
                            assert (lengthLeaf <= options_outputLineLength);
                            unparser_outputindent ((options_outputLineLength - lengthLeaf));
                        }
                        lstringcpy (&(lstringchar (unparser_outputline, unparser_lineLength + 1)), *leafName);
                        unparser_lineLength += lengthLeaf;
                        unparser_emptyLine = false;
                        unparser_lastLeafNameEnd = lstringchar (unparser_outputline, unparser_lineLength);
                    }
                    break;
                }
        }
    }
}

void unparser_printLeaves (const treePT subtreeTP, const int outstream, const bool nl)
{
    try {
        unparser_indent = 0;
        lstringcpy (unparser_outputline, "");
        unparser_lineLength = 0;
        unparser_emptyLine = true;
        unparser_blankLine = true;
        unparser_spacing = true;
        unparser_lastLeafNameEnd = ' ';

        if (options_outputLineLength > maxLineLength) {
            options_outputLineLength = maxLineLength; 
        }

        unparser_maxIndent =  min (1024, (options_outputLineLength / 2));

        unparser_printLeavesTraversal (subtreeTP, outstream);

        if (!unparser_emptyLine) {
            fprintf (tffile (outstream), "%s", unparser_outputline);
            if (nl) {
                fprintf (tffile (outstream), "\n");
            }
        }
    } catch {
        if (exception == STACKLIMIT) {
            error ("", "Output recursion limit exceeded (probable cause: small size or stack limit)", LIMIT_FATAL, 959);
        }
        throw (exception);
    }
}

void unparser_printMatch (const treePT subtreeTP, const treePT matchTreeTP, const int outstream, const bool nl)
{
    unparser_indent = 0;
    lstringcpy (unparser_outputline, "");
    unparser_lineLength = 0;
    unparser_emptyLine = true;
    unparser_blankLine = true;
    unparser_spacing = true;
    unparser_lastLeafNameEnd = ' ';

    if (options_outputLineLength > maxLineLength) {
        options_outputLineLength = maxLineLength;
    }

    unparser_maxIndent =  min (1024, (options_outputLineLength / 2));

    unparser_printMatchTraversal (subtreeTP, matchTreeTP, outstream);

    if (!unparser_emptyLine) {
        fprintf (tffile (outstream), "%s", unparser_outputline);
        if (nl) {
            fprintf (tffile (outstream), "\n");
        }
    }
}


// Function to implement the [quote] and [unparse] predefined externals

static longstring unparser_quotedText;
static int unparser_quotedTextLength;

static void unparser_quoteLeavesTraversal (const treePT subtreeTP)
{
    // Stack use limitation - to avoid crashes
    checkstack ();

    switch (tree_trees[subtreeTP].kind) {

        case treeKind_choose:
            // optimize by skipping choose chains
            {
                treePT chainTP = tree_kids[tree_trees[subtreeTP].kidsKP];
                while (true) {
                    if ((tree_trees[chainTP].kind) != 1) break;
                    chainTP = tree_kids[tree_trees[chainTP].kidsKP];
                }
                unparser_quoteLeavesTraversal (chainTP);
            }
            break;

        case treeKind_order:
            // quote the kids
            {
                kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                for (int i = 1; i <= tree_trees[subtreeTP].count; i++) {
                    if (((tree_trees[tree_kids[subtreeKidsKP]].name) == ATTR_T) && (!(options_option[attr_p]))) break;
                    unparser_quoteLeavesTraversal (tree_kids[subtreeKidsKP]);
                    if (unparser_quotedTextLength == maxLineLength) break;
                    subtreeKidsKP += 1;
                }
            }
            break;

        case treeKind_repeat:
            {
                kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                assert (subtreeKidsKP != nilKid);
                while (true) {
                    unparser_quoteLeavesTraversal (tree_kids[subtreeKidsKP]);
                    if (tree_trees[tree_kids[subtreeKidsKP + 1]].kind == treeKind_empty) break;
                    subtreeKidsKP = tree_trees[tree_kids[subtreeKidsKP + 1]].kidsKP;
                }
            }
            break;

        case treeKind_list:
            {
                kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                if (tree_trees[tree_kids[subtreeKidsKP + 1]].kind != treeKind_empty) {
                    while (true) {
                        unparser_quoteLeavesTraversal (tree_kids[subtreeKidsKP]);
                        subtreeKidsKP = tree_trees[tree_kids[subtreeKidsKP + 1]].kidsKP;
                        if (tree_trees[tree_kids[subtreeKidsKP + 1]].kind == treeKind_empty) break;
                        unparser_quoteLeavesTraversal (commaTP);
                    }
                }
            }
            break;

        case treeKind_empty:
            {
                if (tree_trees[subtreeTP].name == SP_T) {
                    lstringcat (unparser_quotedText, " ");
                    unparser_quotedTextLength += 1;
                } else if ((tree_trees[subtreeTP].name) == SPOFF_T) {
                    unparser_spacing = false;
                } else if ((tree_trees[subtreeTP].name) == SPON_T) {
                    unparser_spacing = true;
                }
            }
            break;

        default :
            {
                // it's a leaf - print it out
                longstring *leafName = ident_idents[tree_trees[subtreeTP].rawname];
                const int lengthLeaf = lstringlen (*leafName);
                const char leafName1 = lstringchar (*leafName, 1);

                if (unparser_spacing && (!unparser_emptyLine)) {
                    if (!(options_option[raw_p])) {
                        if (charset_spaceAfterP[(unsigned char) unparser_lastLeafNameEnd] 
                                && ((charset_spaceBeforeP[(unsigned char) leafName1]) || ((tree_trees[subtreeTP].kind) == treeKind_number))) {
                            if (unparser_quotedTextLength < maxLineLength) {
                                lstringcat (unparser_quotedText, " ");
                                unparser_quotedTextLength += 1;
                            }
                        }
                    } else if (charset_idP[(unsigned char) unparser_lastLeafNameEnd] && charset_idP[(unsigned char) leafName1] && (!(options_option[charinput_p]))) {
                        if (unparser_quotedTextLength < maxLineLength) {
                            lstringcat (unparser_quotedText, " ");
                            unparser_quotedTextLength += 1;
                        }
                    }
                }

                if ((unparser_quotedTextLength + lengthLeaf) > maxLineLength) {
                    error ("", "Result of [quote] predefined function exceeds maximum line length (1048575 characters)", LIMIT_FATAL, 960);
                } else {
                    lstringcat (unparser_quotedText, *leafName);
                    unparser_quotedTextLength += lengthLeaf;
                }

                unparser_lastLeafNameEnd = lstringchar (unparser_quotedText, unparser_quotedTextLength);
                unparser_emptyLine = false;
            }
            break;
    }
}

void unparser_quoteLeaves (const treePT subtreeTP, longstring resultText)
{
    unparser_emptyLine = true;
    unparser_spacing = true;
    unparser_lastLeafNameEnd = ' ';
    lstringcpy (unparser_quotedText, "");
    unparser_quotedTextLength = 0;
    unparser_quoteLeavesTraversal (subtreeTP);
    lstringcpy (resultText, unparser_quotedText);
}


// Procedure to extract the leaves from a tree to implement the [reparse] predefined external

static void unparser_extractLeavesTraversal (const treePT subtreeTP)
{
    // Stack use limitation - to avoid crashes
    checkstack ();

    switch (tree_trees[subtreeTP].kind) {

        case treeKind_choose:
            {
                // optimize by skipping choose chains 
                treePT chainTP = tree_kids[tree_trees[subtreeTP].kidsKP];
                while (true) {
                    if ((tree_trees[chainTP].kind) != 1) break;
                    chainTP = tree_kids[tree_trees[chainTP].kidsKP];
                }
                unparser_extractLeavesTraversal (chainTP);
            }
            break;

        case treeKind_order:
            {
                // reparse the kids
                kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                for (int i = 1; i <= tree_trees[subtreeTP].count; i++) {
                    if (((tree_trees[tree_kids[subtreeKidsKP]].name) == ATTR_T) && (!(options_option[attr_p]))) break;
                    unparser_extractLeavesTraversal (tree_kids[subtreeKidsKP]);
                    subtreeKidsKP += 1;
                }
            }
            break;

        case treeKind_repeat:
            {
                kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                assert (subtreeKidsKP != nilKid);
                while (true) {
                    unparser_extractLeavesTraversal (tree_kids[subtreeKidsKP]);
                    if (tree_trees[tree_kids[subtreeKidsKP + 1]].kind == treeKind_empty) break;
                    subtreeKidsKP = tree_trees[tree_kids[subtreeKidsKP + 1]].kidsKP;
                }
            }
            break;

        case treeKind_list:
            {
                kidPT subtreeKidsKP = tree_trees[subtreeTP].kidsKP;
                assert (subtreeKidsKP != nilKid);
                if (tree_trees[tree_kids[subtreeKidsKP + 1]].kind != treeKind_empty) {
                    while (true) {
                        unparser_extractLeavesTraversal (tree_kids[subtreeKidsKP]);
                        subtreeKidsKP = tree_trees[tree_kids[subtreeKidsKP + 1]].kidsKP;
                        if (tree_trees[tree_kids[subtreeKidsKP + 1]].kind == treeKind_empty) break;
                        unparser_extractLeavesTraversal (commaTP);
                    }
                }
            }
            break;

        case treeKind_empty:
            // do nothing
            break;

        case treeKind_id: case treeKind_upperlowerid: case treeKind_upperid: case treeKind_lowerupperid: case treeKind_lowerid:
            {
                // typed id's become untyped on reparse 
                lastTokenIndex += 1;
                struct tokenTableT *inputToken = &(inputTokens[lastTokenIndex]);
                inputToken->token = tree_trees[subtreeTP].name;
                inputToken->rawtoken = tree_trees[subtreeTP].rawname;
                inputToken->kind = treeKind_id;
            }
            break;

        case treeKind_number: case treeKind_floatnumber: case treeKind_decimalnumber: case treeKind_integernumber:
            {
                // typed numbers become untyped on reparse 
                lastTokenIndex += 1;
                struct tokenTableT *inputToken = &(inputTokens[lastTokenIndex]);
                inputToken->token = tree_trees[subtreeTP].name;
                inputToken->rawtoken = tree_trees[subtreeTP].rawname;
                inputToken->kind = treeKind_number;
            }
            break;

        case treeKind_literal:
            {
                // literals revert to their scan type on reparse 
                lastTokenIndex += 1;
                struct tokenTableT *inputToken = &(inputTokens[lastTokenIndex]);
                inputToken->token = tree_trees[subtreeTP].name;
                inputToken->rawtoken = tree_trees[subtreeTP].rawname;
                inputToken->kind = ident_identKind[tree_trees[subtreeTP].name];
                
                // correct for residual TXL keyword bug - in the long run, 
                // perhaps this can be fixed by un-keying TXL keywords in the ident table 
                if ((inputTokens[lastTokenIndex].kind == treeKind_key) && (!scanner_keyP (tree_trees[subtreeTP].name))) {
                    inputTokens[lastTokenIndex].kind = treeKind_id;
                }
            }
            break;

        default :
            {
                // it's some other kind of leaf - add it to the tokens array 
                lastTokenIndex += 1;
                struct tokenTableT *inputToken = &(inputTokens[lastTokenIndex]);
                inputToken->token = tree_trees[subtreeTP].name;
                inputToken->rawtoken = tree_trees[subtreeTP].rawname;
                inputToken->kind = tree_trees[subtreeTP].kind;
            }
            break;
    }
}

void unparser_extractLeaves (const treePT subtreeTP)
{
    // Extracts the leaves of the given parse tree to the inputTokens array,
    // for use by the [reparse] built-in function
    lastTokenIndex = 0;

    unparser_extractLeavesTraversal (subtreeTP);

    lastTokenIndex += 1;
    inputTokens[lastTokenIndex].token = empty_T;
    inputTokens[lastTokenIndex].kind = treeKind_empty;

    if (options_option[tokens_p]) {
        fprintf (stderr, "\n[reparse] tokens:\n");
        for (int i = 1; i <= lastTokenIndex - 1; i++) {
            fprintf (stderr, "%s '%s'\n", *ident_idents[kindType[inputTokens[i].kind]], *ident_idents[inputTokens[i].token]);
        }
    }
}

// Initialization
void unparser (void) {
    // Line and indent limits
    if (options_outputLineLength > maxLineLength) {
        options_outputLineLength = maxLineLength;
    }

    unparser_maxIndent =  min (1024, (options_outputLineLength / 2));

    unparser_nparselines = 0;
    unparser_localVarsAddress = (struct ruleLocalsT *) 0;

#ifndef NOCOMPILE
    // grammar recursion management - 1-indexed [1 .. maxSymbols]
    arrayalloc (maxSymbols + 1, treePT, unparser_grammarList);
    unparser_grammarList[0] = UNUSED;
    unparser_grammarLength = 0;
#endif

    // indentation
    unparser_tempIndent =  max (1, options_indentIncrement / 2);
    unparser_indent = 0;

    // output line buffers
    unparser_outputLineBufferSize = maxLineLength + maxStringLength + 1;
    arrayalloc (unparser_outputLineBufferSize, char, unparser_savedoutputline);
    arrayalloc (unparser_outputLineBufferSize, char, unparser_outputline);
    unparser_lineLength = 0;

    // output state
    unparser_emptyLine = true;
    unparser_blankLine = true;
    unparser_spacing = true;
    unparser_lastLeafNameEnd = ' ';

    // [quote] management
    unparser_quotedTextLength = 0;
}
